<?php

class LightFeedback {
    
    public $sourceId = null;
    public $submittedDate = null;
    public $retrievedDate = null;
    public $name = null;
    public $title = null;
    public $content = null;
    public $rating = null;
    public $version = null;
    public $deviceType = null;
    
    public function createFeedbackItem() {
        $feedback = new Feedback;
        $feedback->setSourceId($this->sourceId);
        $feedback->setSubmittedDate($this->submittedDate);
        $feedback->setRetrievedDate($this->retrievedDate);
        $feedback->setName($this->name);
        $feedback->setTitle($this->title);
        $feedback->setContent($this->content);
        $feedback->setRating($this->rating);
        $feedback->setVersion($this->version);
        $feedback->setDeviceType($this->deviceType);
        return $feedback;
    }
    
    public function isDuplicateOf(Feedback $feedback) {
        if ($feedback->getSourceId() != $this->sourceId) {
            return false;
        }
        
        if ($feedback->Source->getProvider() == "twitter") {
            $result = (
                $feedback->getSubmittedDate() == $this->submittedDate
                && $feedback->getName() == $this->name
                && $feedback->getContent() == $this->content
            );
        }
        else {
            $result = (
                $feedback->getSubmittedDate() == $this->submittedDate
                && $feedback->getName() == $this->name
                && $feedback->getTitle() == $this->title
                && $feedback->getContent() == $this->content
            );
        }
        
        return $result;
    }
    
    public function getUniqueKey() {
        return $this->submittedDate . $this->version . $this->name . $this->title . $this->content;
    }
}
 
class SourceUpdates {
    
    // We won't allow updates for a source which has another update running for the same provider
    protected static function checkShouldAllowUpdate($provider) {
        $query = Doctrine_Core::getTable('SourceRefreshRun')
            ->createQuery('srr')
            ->innerJoin('srr.Source s')
            ->where('srr.start_date IS NOT NULL')
            ->andWhere('srr.end_date IS NULL')
            ->andWhere('s.provider = ?', $provider);
        
        if ($excludeRunId) {
            $query->andWhere('srr.id != ?', $excludeRunId);
        }
        
        return ($query->count() == 0);
    }
    
    public static function completeRun($run) {
        $retVal = array(
            "notFound" => false,
            "attempted" => false,
            "alreadyRunning" => false,
            "completed" => false,
            "error" => false,
            "run" => null,
            "addedCount" => 0,
        );
        
        // Do the final concurrent updates check before running
        if (!self::checkShouldAllowUpdate($run->Source->getProvider())) {
            $retVal["alreadyRunning"] = true;
            LogHelper::log('Has currently running query');
            return $retVal;
        }
        
        // OK, we're good, do the update
        return self::performRun($run);
    }
    
    public static function newRun($sourceId, $userId = null) {
        $retVal = array(
            "notFound" => false,
            "attempted" => false,
            "alreadyRunning" => false,
            "completed" => false,
            "error" => false,
            "run" => null,
            "addedCount" => 0,
        );
        
        LogHelper::log('New run for source ID ' . $sourceId);
        
        if (!$sourceId) {
            $retVal["notFound"] = true;
            LogHelper::log('No source ID provided');
            return $retVal;
        }
        
        $source = Doctrine_Core::getTable('Source')->find($sourceId);
        if (!$source) {
            $retVal["notFound"] = true;
            LogHelper::log('Invalid source ID provided');
            return $retVal;
        }
    
        if (!self::checkShouldAllowUpdate($source->getProvider())) {
            $retVal["alreadyRunning"] = true;
            LogHelper::log('Has currently running query');
            return $retVal;
        }
        
        $run = new SourceRefreshRun;
        
        if ($userId) {
            $run->setUserInitiatedId($userId);
        }
        
        $run->setSourceId($source->getId());
        $run->save();
        
        LogHelper::log('Created SourceRefreshRun');
        
        // Here's where the guts of the update happens
        $retVal = self::performRun($run);
        
        $retVal["run"] = $run;
        
        return $retVal;
    }
    
    protected static function performRun($run) {
        $retVal = array(
            "notFound" => false,
            "attempted" => false,
            "alreadyRunning" => false,
            "completed" => false,
            "error" => false,
            "source" => null,
            "addedCount" => 0,
        );
        
        $markedCompletedSuccessfully = false;
        
        try {
            LogHelper::log('Starting run ' . $run->getId() . ' for source ' . $run->Source->getName());
            
            $retVal["attempted"] = true;
            
            $run->setStartDate(date("Y-m-d H:i:s"));
            $run->save();
            
            // Update the source itself
            
            SourceUpdates::updateSourceRecord($run->Source);
            
            // Update the feedback
            
            $sourceLastFeedbackItems = $run->Source->getLastFeedbackItems();
            if ($sourceLastFeedbackItems) {
                foreach ($sourceLastFeedbackItems as $sourceLastFeedbackItem) {
                    LogHelper::log('sourceLastFeedbackItem was ID = "' . $sourceLastFeedbackItem->getId() . '", name = "' . $sourceLastFeedbackItem->getName() . '", title = "' . $sourceLastFeedbackItem->getTitle() . '", content = "' . $sourceLastFeedbackItem->getContent() . '", date = "' . $sourceLastFeedbackItem->getSubmittedDate() . '", version = "' . $sourceLastFeedbackItem->getVersion() . '".');
                }
            }
            else {
                LogHelper::log('sourceLastFeedbackItem(s) was NULL');
            }
            
            $addedCount = 0;
            
            // Actually retrieve the feedback...this can be big
            $newFeedbackItems = self::getFeedbackForSource($run->Source, $sourceLastFeedbackItems, $run);
            LogHelper::log('Feedback collection complete for run ' . $run->getId());
            
            if ($newFeedbackItems && is_array($newFeedbackItems) && count($newFeedbackItems) > 0) {
                
                // getFeedbackForSource can be slow. Let's ensure our connection is still current
                Doctrine_Manager::connection()->connect();
                Doctrine_Manager::connection()->setAttribute(Doctrine_Core::ATTR_AUTO_FREE_QUERY_OBJECTS, true);
                
                $newFeedbackItems = array_reverse($newFeedbackItems); // Reverse the order so that the IDs are in the correct order
                foreach ($newFeedbackItems as $newFeedbackItem) {
                    
                    $feedback = $newFeedbackItem->createFeedbackItem();
                    $feedback->setAddedSourceRefreshRunId($run->getId());
                    LogHelper::log('Saving feedback item with sourceId = "' . $feedback->getSourceId() . '", name = "' . $feedback->getName() . '", title = "' . $feedback->getTitle() . '", content = "' . $feedback->getContent() . '", rating = "' . $feedback->getRating() . '", version = "' . $feedback->getVersion() . '", deviceType = "' . $feedback->getDeviceType() . '", submittedDate = "' . $feedback->getSubmittedDate() . '", retrievedDate = "' . $feedback->getRetrievedDate() . '", addedSourceRefreshRunId = "' . $feedback->getAddedSourceRefreshRunId() . '".');
                    $feedback->save();
                    
                    $feedback->free();
                    unset($feedback);
                    
                    $addedCount++;
                }
            }
            
            $retVal["addedCount"] = $addedCount;
            
            // Finalise the SourceRefreshRun
            
            $run->setEndDate(date("Y-m-d H:i:s"));
            $run->setCompletedSuccessfully(true);
            $run->save();
            $markedCompletedSuccessfully = true;
            
            $retVal["completed"] = true;
            
            LogHelper::log('Finished SourceRefreshRun');
            
            // For debugging...
            
            $duplicateCount = Doctrine_Query::create()
                ->select('COUNT(*), f.title, f.content, f.name, f.submitted_date')
                ->from('Feedback f')
                ->where('f.source_id = ?', $run->Source->getId())
                ->groupBy("f.title, f.content, f.name, f.submitted_date")
                ->having("COUNT(*) > 1")
                ->count();
            if ($duplicateCount > 0) {
                LogHelper::log('Oh no! There are now ' . $duplicateCount . ' duplicates!');
            }
            else {
                LogHelper::log('No duplicates.');
            }
        }
        catch (Exception $e) {
            LogHelper::log('Exception caught');
            LogHelper::log($e);
            
            if (!$markedCompletedSuccessfully) {
                $run->setEndDate(date("Y-m-d H:i:s"));
                $run->save();
            }
            
            $retVal["error"] = true;
        }
        
        return $retVal;
    }
    
    public static function getFeedbackForSource($source, $lastFeedbackItems, $run) {
        if ($source->getProvider() == "itunes") {
            return self::getItunesFeedbackForSource($source, $lastFeedbackItems, $run);
        }
        else if ($source->getProvider() == "play") {
            return self::getPlayFeedbackForSource($source, $lastFeedbackItems, $run);
        }
        else if ($source->getProvider() == "twitter") {
            return self::getTwitterFeedbackForSource($source, $lastFeedbackItems, $run);
        }
        
        return array();
    }
    
    protected static function getItunesFeedbackForSource($source, $lastFeedbackItems, $run) {
        
        $config = json_decode($source->getProviderConfig(), true);
        if (!isset($config["id"])) {
            LogHelper::log("A iTunes ID must be specified.");
            die("A iTunes ID must be specified.");
        }
        
        $feedbackItems = array();
        $addAnotherRunToMakeUpForDroppedFeedbackItems = false;
        
        $keepLooking = true;
        $currentPageNumber = 0;
        while ($keepLooking) {
            
            // Necessary to keep our connection alive for long-running updates
            $run->updateLastActivityDate();
            
            $countryConfig = Config::$countries[$config["country"]];
            
            $url = "https://itunes.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?id=" . $config["id"] . "&pageNumber=" . $currentPageNumber . "&sortOrdering=0&type=Purple+Software";
            
            LogHelper::log("Requesting: " . $url);
            
            $ch = curl_init();
            curl_setopt($ch, CURLOPT_URL, $url);
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
            curl_setopt($ch, CURLOPT_HTTPHEADER, array(
                "User-Agent: iTunes/9.2 (Macintosh; U; PPC Mac OS X 10.6)",
                "X-Apple-Store-Front: " . $countryConfig["code"] . "-1",
            ));
            $body = curl_exec($ch);
            curl_close($ch);
            
            LogHelper::log("HTTP response code: " . $httpStatus . ", response size: " . strlen($body));
            
            $dom = new DOMDocument();
            @$dom->loadHTML($body);
            $xpath = new DOMXPath($dom);
            $reviews = $xpath->query("//document/view/scrollview/vboxview/view/matrixview/vboxview/vboxview/vboxview");
        
            if ($reviews->length > 0) {
            
                for ($i = 0; $i < $reviews->length; $i++) {
                    
                    $review = $reviews->item($i);
                    if ($review) {
                        
                        $title = $xpath->query("hboxview/textview/setfontstyle/b", $review);
                        $user = $xpath->query("hboxview/textview/setfontstyle/gotourl/b", $review);
                        $rating = $xpath->query("hboxview/hboxview/hboxview", $review);
                        $versionAndDate = $xpath->query("hboxview/textview/setfontstyle", $review);
                        $content = $xpath->query("textview/setfontstyle", $review);
                        
                        // The content of $user is a subset of $versionAndDate, and it can contain and undetermined number of hyphens...safest to just remove this content before exploding
                        $versionAndDateBits = explode("-", str_replace(trim($user->item(0)->textContent), "", $versionAndDate->item(1)->textContent)); 
                        
                        // Can't handle anonymous feedback items - they don't have a date! Messes up everything!
                        // TODO Should really find a way around this...
                        if (trim($versionAndDate->item(1)->textContent) != "by Anonymous") {
                            
                            $feedbackItem = new LightFeedback;
                            $feedbackItem->sourceId = $source->getId();
                            $feedbackItem->submittedDate = date("Y-m-d H:i:s", strtotime(TextHelper::escape($versionAndDateBits[2] . " " . $versionAndDateBits[3] . " " . $versionAndDateBits[4])));
                            $feedbackItem->retrievedDate = date("Y-m-d H:i:s");
                            $feedbackItem->version = TextHelper::escape(str_replace("Version", "", $versionAndDateBits[1]));
                            $feedbackItem->name = TextHelper::escape($user->item(0)->textContent);
                            $feedbackItem->title = TextHelper::escape($title->item(0)->textContent);
                            $feedbackItem->content = TextHelper::escape($content->item(0)->textContent);
                            $feedbackItem->rating = TextHelper::escape(str_replace("stars", "", $rating->item(0)->attributes->getNamedItem("alt")->value));
                            
                            // Before we add it, check if it's not the same as one of the most recently added feedback. If it is, stop.
                            if ($lastFeedbackItems) {
                                foreach ($lastFeedbackItems as $lastFeedbackItem) {
                                    if ($feedbackItem->isDuplicateOf($lastFeedbackItem)) {
                                        $keepLooking = false;
                                        break;
                                    }
                                }
                                if ($keepLooking == false) {
                                    break;
                                }
                            }
                            
                            // Use the unique key as the array key to ensure we don't add the same record twice - this can happen
                            // when a new review is added between page retrievals
                            $feedbackItems[$feedbackItem->getUniqueKey()] = $feedbackItem;
                            
                            // We can't process more than Config::$maxRecordsPerUpdate records at once or we'll get errors. But we can't stop adding records because
                            // we're adding later records, and they'll be missed later. So shift items off the start of the array - the next update will add them
                            // in a separate process.
                            if (count($feedbackItems) > Config::$maxRecordsPerUpdate) {
                                $feedbackItemToDrop = array_shift($feedbackItems);
                                unset($feedbackItemToDrop);
                                
                                $addAnotherRunToMakeUpForDroppedFeedbackItems = true;
                            }
                        }
                    }   
                }
                
                $currentPageNumber++;
            }
            else {
                $keepLooking = false;
            }            
        }
        
        if ($addAnotherRunToMakeUpForDroppedFeedbackItems) {
            $anotherRun = new SourceRefreshRun;
            $anotherRun->setSourceId($source->getId());
            $anotherRun->setRequestedDate(date("Y-m-d H:i:s"));
            $anotherRun->save();
        }
        
        return $feedbackItems;
    }
    
    protected static function getPlayFeedbackForSource($source, $lastFeedbackItems, $run) {
        
        $config = json_decode($source->getProviderConfig(), true);
        if (!isset($config["id"])) {
            LogHelper::log("A Google Play ID must be specified.");
            die("A Google Play ID must be specified.");
        }
        
        LogHelper::log("Getting Google Play feedback");
        
        $feedbackItems = array();
        $addAnotherRunToMakeUpForDroppedFeedbackItems = false;
        
        $keepLooking = true;
        $duplicateFound = false;
        $currentPageNumber = 0;
        $lastCallContents = "";
        while ($keepLooking && !$duplicateFound) {
            $keepLooking = false;
            
            // Necessary to keep our connection alive for long-running updates
            $run->updateLastActivityDate();
            
            // Necessary to not trigger Google's bot detection
            sleep(6); 
            
            LogHelper::log("Requesting: " . "https://play.google.com/store/getreviews?id=" . $config["id"] . "&reviewType=0&reviewSortOrder=0&pageNum=" . $currentPageNumber . "&xhr=1");
            
            //$page = file_get_contents("/Users/cameronross/Projects/Coverd/mockGooglePlayResponse.2014-05-02.065729.txt");
            
            $ch = curl_init();
            curl_setopt($ch, CURLOPT_URL, "https://play.google.com/store/getreviews");
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
            curl_setopt($ch, CURLOPT_POST, 1);
            curl_setopt($ch, CURLOPT_POSTFIELDS, "id=" . $config["id"] . "&reviewType=0&reviewSortOrder=0&pageNum=" . $currentPageNumber . "&xhr=1");
            curl_setopt($ch, CURLOPT_HTTPHEADER , array(
                "Accept-Language: en-US,en;q=0.8",
                "Content-Type: application/x-www-form-urlencoded;charset=UTF-8",
                "Referer: https://play.google.com/store/apps/details?id=" . $config["id"],
                "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36",
            ));
            curl_setopt($ch, CURLOPT_ENCODING, 'gzip,deflate,sdch'); 
            $page = curl_exec($ch);
            $httpStatus = curl_getinfo($ch, CURLINFO_HTTP_CODE);
            curl_close($ch);
            //file_put_contents("/Users/cameronross/Projects/Coverd/mockGooglePlayResponse." . date("Y-m-d.His") . ".txt", $page);
            
            $cleanedPageContents = substr($page, 4);
            $json = json_decode($cleanedPageContents, true);
            
            // Simple check to prevent from looping infinitely on the last page (not sure why this is...)
            if ($lastCallContents != $json[0][2]) {
                
                $dom = new DOMDocument();
                @$dom->loadHTML($json[0][2]);
                $xpath = new DOMXPath($dom);
                $reviews = $xpath->query("//div[@class='single-review']");
                
                LogHelper::log("Found " . $reviews->length . " comments");
                
                if ($reviews->length > 0) {
                    
                    for ($i = 0; $i < $reviews->length; $i++) {
                        $review = $reviews->item($i);
                        
                        //LogHelper::log("Deconstructing review with textContent = '" . $review->textContent . "'");
                        
                        $title = "";
                        $titleNodes = $xpath->query("div[@class='review-body']/span[@class='review-title']", $review);
                        if ($titleNodes->length > 0) {
                            $title = $titleNodes->item(0)->textContent;
                        }
                        
                        $name = "";
                        $nameNodes = $xpath->query("div[@class='review-header']/div[@class='review-info']/span[@class='author-name']/a", $review);
                        if ($nameNodes->length > 0) {
                            $name = $nameNodes->item(0)->textContent;
                        }
                        
                        // We need to have a date
                        $interpretedDate = 0;
                        $dateNodes = $xpath->query("div[@class='review-header']/div[@class='review-info']/span[@class='review-date']", $review);
                        if ($dateNodes->length > 0) {
                            $interpretedDate = strtotime(TextHelper::escape($dateNodes->item(0)->textContent));
                        }
                        if ($interpretedDate == 0) {
                            // Sometimes this interpretation fails, not sure why. But let's not save a 0 date, next best thing is set the time to now.
                            // TODO Look more into this
                            $interpretedDate = time();
                        }
                        
                        $rating = 0;
                        $ratingNodes = $xpath->query("div[@class='review-header']/div[@class='review-info']/div[@class='review-info-star-rating']/div[@class='tiny-star star-rating-non-editable-container']/div[@class='current-rating']", $review);
                        if ($ratingNodes->length > 0) {
                            $cssPercentage = $ratingNodes->item(0)->attributes->getNamedItem("style");
                            if ($cssPercentage && $cssPercentage->value) {
                                $cssPercentage = trim(str_replace("width: ", "", str_replace("%", "", $cssPercentage->value)));
                                if ($cssPercentage > 0) {
                                    $rating = $cssPercentage / 20;
                                }
                            }
                        }
                        
                        $content = "";
                        $bodyNodes = $xpath->query("div[@class='review-body']", $review);
                        if ($bodyNodes->length > 0) {
                            $content = trim($bodyNodes->item(0)->textContent);
                            
                            if ($title) {
                                $content = trim(substr($content, strlen($title)));
                            }
                            
                            $reviewLinkText = "Full Review";
                            if (substr($content, (-1 * strlen($reviewLinkText))) == $reviewLinkText) {
                                $content = trim(substr($content, 0, (-1 * strlen($reviewLinkText))));
                            }
                        }
                    
                        $feedbackItem = new LightFeedback;
                        $feedbackItem->sourceId = $source->getId();
                        $feedbackItem->submittedDate = date("Y-m-d H:i:s", $interpretedDate);
                        $feedbackItem->retrievedDate = date("Y-m-d H:i:s");
                        //$feedbackItem->version = TextHelper::escape($appVersion);
                        $feedbackItem->name = TextHelper::escape($name);
                        $feedbackItem->title = TextHelper::escape($title);
                        $feedbackItem->content = TextHelper::escape($content);
                        $feedbackItem->rating = TextHelper::escape($rating);
                        //$feedbackItem->deviceType = TextHelper::escape($device);
                        
                        //LogHelper::log("Extracted row content = " . print_r($feedbackItem, true));
                    
                        // Before we add it, check if it's not the same as one of the most recently added feedback. If it is, stop.
                        if ($lastFeedbackItems) {
                            foreach ($lastFeedbackItems as $lastFeedbackItem) {
                                if ($feedbackItem->isDuplicateOf($lastFeedbackItem)) {
                                    $duplicateFound = true;
                                    break;
                                }
                            }
                            if ($duplicateFound == true) {
                                break;
                            }
                        }
                        
                        // Use the unique key as the array key to ensure we don't add the same record twice - this can happen
                        // when a new review is added in the X second sleep between page retrievals
                        $feedbackItems[$feedbackItem->getUniqueKey()] = $feedbackItem;
                        
                        // We can't process more than Config::$maxRecordsPerUpdate records at once or we'll get errors. But we can't stop adding records because
                        // we're adding later records, and they'll be missed later. So shift items off the start of the array - the next update will add them
                        // in a separate process.
                        if (count($feedbackItems) > Config::$maxRecordsPerUpdate) {
                            $feedbackItemToDrop = array_shift($feedbackItems);
                            unset($feedbackItemToDrop);
                            
                            $addAnotherRunToMakeUpForDroppedFeedbackItems = true;
                        }
                    }
                    
                    LogHelper::log("Finished processing all items in this call");
                    
                    $currentPageNumber++;
                    $keepLooking = true;
                }
            }
            
            $lastCallContents = $json[0][2];
        }
        
        if ($addAnotherRunToMakeUpForDroppedFeedbackItems) {
            $anotherRun = new SourceRefreshRun;
            $anotherRun->setSourceId($source->getId());
            $anotherRun->setRequestedDate(date("Y-m-d H:i:s"));
            $anotherRun->save();
        }
        
        return $feedbackItems;
    }
    
    protected static function getTwitterFeedbackForSource($source, $lastFeedbackItems, $run) {
        $config = json_decode($source->getProviderConfig(), true);
        if (!isset($config["term"])) {
            LogHelper::log("A Twitter search term must be specified.");
            die("A Twitter search term must be specified.");
        }
        
        $feedbackItems = array();
        
        // Authentication
        
        $consumerKey = "QeVMKJv0chHAlgK0FL1dOA";
        $consumerSecret = "e8mUi4PinH6iOcCqjYBNOr7ZWSHOVwjrA6vbRARYtM";
        
        $encodedBearerToken = base64_encode(urlencode($consumerKey) . ":" . urlencode($consumerSecret));
        
        $authHandle = curl_init();
        curl_setopt($authHandle, CURLOPT_URL, "https://api.twitter.com/oauth2/token");
        curl_setopt($authHandle, CURLOPT_HTTPHEADER, array(
            'Authorization: Basic ' . $encodedBearerToken,
            'Content-Type: application/x-www-form-urlencoded;charset=UTF-8'
        ));
        curl_setopt($authHandle, CURLOPT_POSTFIELDS, "grant_type=client_credentials");
        curl_setopt($authHandle, CURLOPT_RETURNTRANSFER,true);
        $rawResult = curl_exec($authHandle);
        curl_close($authHandle);
        
        $result = json_decode($rawResult, true);
        
        if ($result && is_array($result) && $result["token_type"] == "bearer") {
            $bearerToken = $result["access_token"];
            //echo "bearerToken = " . $bearerToken . PHP_EOL;
            
            // All authenticated, now start retrieving tweets
        
            $keepLooking = true;
            $duplicateFound = false;
            $lastStatusId = null;
            
            while ($keepLooking && !$duplicateFound) {
                $keepLooking = false;
                
                $url = "https://api.twitter.com/1.1/search/tweets.json?count=100&include_entities=true&result_type=recent&lang=en&q=" . urlencode($config["term"]) . ($lastStatusId ? "&max_id=" . $lastStatusId : "");
                //echo "url = " . $url . PHP_EOL;
                
                $searchHandle = curl_init();
                curl_setopt($searchHandle, CURLOPT_URL, $url);
                curl_setopt($searchHandle, CURLOPT_HTTPHEADER, array(
                    'Authorization: Bearer ' . $bearerToken,
                ));
                curl_setopt($searchHandle, CURLOPT_RETURNTRANSFER,true);
                $data = curl_exec($searchHandle);
                curl_close($searchHandle);
                
                //echo $data;
                
                if ($data) {
                    $jsonData = json_decode($data, true);
                    if ($jsonData && is_array($jsonData)) {
                        
                        //echo "count of responses = " . count($jsonData["statuses"]) . PHP_EOL;
                        
                        if (is_array($jsonData["statuses"]) && count($jsonData["statuses"]) > 0) {
                            
                            foreach ($jsonData["statuses"] as $result) {
                                
                                //echo "processing item with ID " .  $result["id_str"] . PHP_EOL;
                                
                                $feedbackItem = new LightFeedback;
                                $feedbackItem->sourceId = $source->getId();
                                $feedbackItem->submittedDate = date("Y-m-d H:i:s", strtotime($result["created_at"]));
                                $feedbackItem->retrievedDate = date("Y-m-d H:i:s");
                                $feedbackItem->name = TextHelper::escape($result["user"]["screen_name"]);
                                $feedbackItem->content = TextHelper::escape($result["text"]);
                            
                                // Before we add it, check if it's not the same as one of the most recently added feedback. If it is, stop.
                                if ($lastFeedbackItems) {
                                    foreach ($lastFeedbackItems as $lastFeedbackItem) {
                                        if ($feedbackItem->isDuplicateOf($lastFeedbackItem)) {
                                            $duplicateFound = true;
                                            break;
                                        }
                                    }
                                    if ($duplicateFound == true) {
                                        break;
                                    }
                                }
                                
                                $feedbackItems[$feedbackItem->getUniqueKey()] = $feedbackItem;
                                
                                $lastStatusId = $result["id_str"];
                            }
                            
                            $currentPageNumber++;
                            $keepLooking = true;
                        }
                        
                        // Because of Twitter's max_id approach of paginating, if we get to the end of the list of tweets, the loop can endlessly get
                        // one result, returning the same one last tweet. Protect against this.
                        if ($lastStatusId != null && count($jsonData["statuses"]) == 1) {
                            $keepLooking = false;
                        }
                    }
                }
            }
        }
        
        return $feedbackItems;
    }
    
    public static function updateSourceRecord($source) {
        LogHelper::log("updateSourceRecord: source is " . $source->getProvider());
        
        if ($source->getProvider() == "itunes") {
            self::updateItunesSource($source);
        }
        else if ($source->getProvider() == "play") {
            self::updateGooglePlaySource($source);
        }
    }
    
    protected static function updateItunesSource($source) {
        LogHelper::log("updateItunesSource: in");
        
        $config = json_decode($source->getProviderConfig(), true);
        if (!isset($config["id"])) {
            LogHelper::log("A iTunes ID must be specified.");
            die("A iTunes ID must be specified.");
        }
        
        LogHelper::log("updateItunesSource: setting up URL");
        
        $countryConfig = Config::$countries[$config["country"]];
        
        $url = "http://itunes.apple.com/lookup?country=" . $countryConfig["itunes"] . "&id=" . $config["id"];
        
        LogHelper::log("updateItunesSource: url = " . $url);
        
        $data = file_get_contents($url);
        if ($data) {
            $jsonData = json_decode($data, true);
            if ($jsonData && is_array($jsonData) && $jsonData["resultCount"] >= 1 && $jsonData["results"][0]) {
                $sourceRecord = $jsonData["results"][0];
                $requiresSave = false;
                
                // Check for icon (if we don't already have one)
                if (!$source->getIconUrl()) {
                    if (isset($sourceRecord["artworkUrl100"])) {
                        $imageWebPath = ImageHelper::saveLocalVersionOfImage($sourceRecord["artworkUrl100"]);
                        $source->setIconUrl($imageWebPath);
                        $requiresSave = true;
                    }
                }
                
                // Check for name update
                if (isset($sourceRecord["trackName"])) {
                    $newName = $sourceRecord["trackName"];
                    if ($source->getName() != $newName) {
                        $source->setName($newName);
                        $requiresSave = true;
                    }
                }
                
                // Check for rating update
                if (isset($sourceRecord["averageUserRating"])) {
                    $newRating = $sourceRecord["averageUserRating"];
                    if ($source->getRating() != $newRating) {
                        $source->setRating($newRating);
                        $requiresSave = true;
                    }
                }
                
                if ($requiresSave) {
                    $source->save();
                }
            }
        }
        
        LogHelper::log("updateItunesSource: done");
    }
    
    protected static function updateGooglePlaySource($source) {
        $config = json_decode($source->getProviderConfig(), true);
        if (!isset($config["id"])) {
            LogHelper::log("A Google Play ID must be specified.");
            die("A Google Play ID must be specified.");
        }
        
        $data = file_get_contents("https://42matters.com/api/1/apps/lookup.json?access_token=dfeb9847ecc944eac21c33d604c46aeb4885b1fc&p=" . $config["id"]);
        if ($data) {
            $jsonData = json_decode($data, true);
            if ($jsonData && is_array($jsonData) && $jsonData["package_name"]) {
                $requiresSave = false;
                
                // Check for icon (if we don't already have one)
                if (!$source->getIconUrl()) {
                    if (isset($jsonData["icon"])) {
                        $imageWebPath = ImageHelper::saveLocalVersionOfImage($jsonData["icon"]);
                        $source->setIconUrl($imageWebPath);
                        $requiresSave = true;
                    }
                }
                
                // Check for name update
                if (isset($jsonData["title"])) {
                    $newName = $jsonData["title"];
                    if ($source->getName() != $newName) {
                        $source->setName($newName);
                        $requiresSave = true;
                    }
                }
                
                // Check for rating update
                if (isset($jsonData["rating"])) {
                    $newRating = $jsonData["rating"];
                    if ($source->getRating() != $newRating) {
                        $source->setRating($newRating);
                        $requiresSave = true;
                    }
                }
                
                if ($requiresSave) {
                    $source->save();
                }
            }
        }
    }
    
}